#include <assert.h>
#include <getopt.h>
#include <stdio.h>
#include <map>
#include <vector>
#include <algorithm>


#include "idngram.h"
#include "idngram_merge.h"


static struct option long_options[] =
{
	{"NMax", 1, 0, 'n'},
	{"out", 1, 0, 'o'},
	{"swap", 1, 0, 's'},
	{"para", 1, 0, 'p'},
	{0, 0, 0, 0}
};

static int N=0;
static int paraMax=0;
static char* output=NULL;
static char* swapfile=NULL;
static std::vector<long> para_offsets;

template<int N>
void WriteOut(FILE* out, std::map<CIdngram<N>, unsigned int> &map)
{
	typedef typename std::map<CIdngram<N>, unsigned int>::iterator TMapIterator;
	TMapIterator its=map.begin(), ite=map.end();
	for (; its != ite; ++its) {
          fwrite(its->first.ids, sizeof(TWORDID), N, out); // ids 
		fwrite(&(its->second), sizeof(unsigned int), 1, out); // freq!
	}
	map.clear();
}



void ShowUsage()
{
	printf("Usage:\n\tids2idngram options idsfile[ idsfile...]\n");
	printf("\nDescription\n");
	printf("   This program generate idngram file, which is a sorted [id1,..idN,freq] array, from binary id stream files.\n");
	printf("\nInput:\n");
	printf("\tBinary id stream files looks like [id0,...,idX]\n");
	printf("\nOptions:\n");
	printf("\t  -n N               # N-gram\n");
	printf("\t  -s swapfile        # intermedia temporary file\n");
	printf("\t  -o outputfile      # result idngram file [id1, ... idN, freq]*\n");
	printf("\t  -p para_size       # maxium ngram-items per para\n");
	printf("\nExample:\n");
	printf("   Following example will use three input idstream file idsfile[1,2,3] to generate the idngram file all.id3gram. Each para (internal map size or hash size) would be 1024000, using swap file for temp result. All temp para result would final be merged to got the final result.\n");
	printf("\tids2idngram -n 3 -s /tmp/swap -o all.id3gram -p 1024000 idsfile1 idsfile2 idsfile3\n\n");
	exit(100);
}

template <int N>
void ProcessingRead(FILE *fp, FILE *swap, std::vector<long>& para_offsets, size_t paraMax)
{
	typedef CIdngram<N> TNgram;
	typedef typename std::map<CIdngram<N>, unsigned int> TMap;
	
	TMap map;
	TNgram ngram;
	
	TWORDID *ids = ngram.ids;
	fread(ids, sizeof(TWORDID), N-1, fp);
	while (fread(ids+N-1, sizeof(TWORDID), 1, fp) == 1) {
		++map[ngram];
		if (map.size() >= paraMax) {
			printf("."); fflush(stdout);
			WriteOut(swap, map);
			para_offsets.push_back(ftell(swap));
		}
		for (int i = 0; i < N-1; ++i) {
			ids[i] = ids[i+1];
		}
	}
	if (map.size() >= 0) {
		printf("."); fflush(stdout);
		WriteOut(swap, map);
		para_offsets.push_back(ftell(swap));
	}
}



static void getParameters(int argc, char* const argv[])
{
	int option_index = 0;
	int c;
	while ((c=getopt_long(argc, argv, "p:n:s:o:", long_options, &option_index)) != -1)
	{
		switch (c) {
		case 'n':
			N = atoi(strdup(optarg));
			break;
		case 'p':
			paraMax = atoi(strdup(optarg));
			break;
		case 'o':
			output = strdup(optarg);
			break;
		case 's':
			swapfile = strdup(optarg);
			break;
		default:
			ShowUsage();
		}
	}
	if (N < 1 || N > 3 || paraMax < 1024 || output == NULL || swapfile == NULL) 
		ShowUsage();
}

int main(int argc, char* argv[])
{
	getParameters(argc, argv);
	FILE *swap = fopen(swapfile, "wb+");
	FILE *out = fopen(output, "wb+");
	if (optind >= argc) ShowUsage();
	while (optind < argc) {
		printf("Processing %s:", argv[optind]);fflush(stdout);
		FILE *fp = fopen(argv[optind], "rb");
		switch (N) {
		case 1:
			ProcessingRead<1>(fp, swap, para_offsets, paraMax);	
		case 2:
			ProcessingRead<2>(fp, swap, para_offsets, paraMax);
		case 3:
			ProcessingRead<3>(fp, swap, para_offsets, paraMax);
		}
		fclose(fp);
		printf("\n");fflush(stdout);
		++optind;
	}
	printf("Merging...");fflush(stdout);
	switch (N) {
		case 1:
		ProcessingIdngramMerge<1>(swap, out, para_offsets);
		break;
		case 2:
		ProcessingIdngramMerge<2>(swap, out, para_offsets);
		break;
		case 3:
		ProcessingIdngramMerge<3>(swap, out, para_offsets);
		break;
	}
	printf("Done\n");fflush(stdout);
	fclose(out);
	fclose(swap);
	return 0;
}
